# -*- coding: utf-8 -*-
import scrapy
from scrapy_news.items import ScrapyNewsItem
import copy
# from urllib.parse import urljoin
import urlparse
import time
import chardet
import json

class NewsSpider(scrapy.Spider):
    name = 'news'
    allowed_domains = ['news.inewsweek.cn','auto.inewsweek.cn','channel.inewsweek.chinanews.com','www.inewsweek.cn','www.zgxwzk.chinanews.com.cn']
    # start_urls = ['https://www.xuexi.cn/']

    def start_requests(self):  # 由此方法通过下面链接爬取页面

        # 定义爬取的链接
        urls = [
            # 'http://news.inewsweek.cn/',
            'http://news.inewsweek.cn/'
        ]

        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)  # 爬取到的页面如何处理？提交给parse方法处理

    def parse(self, response):
        # self.log('111111111111111111111' )
        # print response.body

        # 一级分类
        cat1 = response.xpath('//*[@id="app"]/div[1]/div/div[2]/ul/li')
        item = ScrapyNewsItem()

        for i,c1 in enumerate(cat1):
            if i < 1 or i > 7 :
                continue

            #一级目录名称
            c1title = c1.xpath("./a/text()")

            item['c1title'] = c1title.extract_first()
            # cat2 = c1.xpath(".//div[@class='product_list_box']")
            url = c1.xpath("./a/@href")
            item['listurl'] = 'http:'+url.extract_first()
            print item['listurl']
            yield scrapy.Request(
                item['listurl'],
                callback=self.parse_list,
                meta={"item": copy.deepcopy(item)}
            )











    #某个分类的列表页面，里面是分类下面的新闻
    def parse_list(self,response):
        # print response.body
        item = response.meta['item']
        metaitem = copy.deepcopy(item)
        newslist = response.xpath('//*[@id="app"]/div[3]/div[@class="grid-item"]')
        # print '11111111111111'
        # print c1
        # print item//*[@id="app"]/div[3]/div[4]
        # print response.url




        for i,news in enumerate(newslist):
            img = news.xpath("./a/img")
            if img:
                newsurl = news.xpath("./a/@href").extract_first()
                imgurl = news.xpath("./a/img/@src").extract_first()
                newsname = news.xpath("./a/div[1]/p/text()").extract_first()
                # print news.xpath("//*[@id="app"]/div[3]/div[5]/a/div[1]/p")
                item['newsurl'] = urlparse.urljoin(item['listurl'],newsurl)
                item['ll_title'] = newsname
                item['ll_pics'] = urlparse.urljoin(item['listurl'],imgurl)

                yield scrapy.Request(
                    item['newsurl'],
                    callback=self.parse_datail,
                    meta={"item":copy.deepcopy(item)}
                )




        #分页
        url_change = urlparse.urlparse(response.url)
        ajaxurl = "http://channel.inewsweek.chinanews.com/cns/lm/{0}.shtml?pager={1}&pagenum={2}"

        for ip in range(2,50):
            print ip
            if url_change.path.strip('/') == '':
                # ajaxurl = ajaxurl.format('auto', ip, 20)  # json 格式的分页，url的拼接
                ajaxurl = "http://channel.inewsweek.chinanews.com/cns/lm/auto.shtml?pager="+str(ip)+"&pagenum=20"
            else:
                ajaxurl = "http://channel.inewsweek.chinanews.com/cns/lm/"+url_change.path.strip('/')+".shtml?pager="+str(ip)+"&pagenum=20"
                # ajaxurl = ajaxurl.format(url_change.path.strip('/'), ip, 20)  # json 格式的分页，url的拼接
            print ajaxurl
            yield scrapy.Request(
                ajaxurl,
                callback=self.parse_json,
                meta={"item": copy.deepcopy(metaitem)}
            )





    #json分页处理
    def parse_json(self,response):
        item = response.meta['item']
        metaitem = copy.deepcopy(item)
        jsonlist = json.loads(response.body.split('\n')[0].split('=')[1].strip(';'))['docs']
        for i in jsonlist:
            item['newsurl'] = i['url']
            item['ll_title'] = i['title']
            item['ll_pics'] = i['img_cns']
            print item['ll_title']
            yield scrapy.Request(
                item['newsurl'],
                callback=self.parse_datail,
                meta={"item": copy.deepcopy(item)}
            )



    #详细页面
    def parse_datail(self,response):
        item = response.meta['item']
        contentdiv = response.xpath("//div[@class='contenttxt']")
        #富文本
        ll_content = contentdiv.extract()

        item['ll_content'] = ll_content
        # item['ll_content'] = ''
        item['ll_state'] = 1
        item['ll_time'] = int(time.time())

        # print  chardet.detect(item['c1title'].encode())

        if item['c1title'] == u'时政':
            item['ll_cid'] = 2
        elif item['c1title'] == u'社会':
            item['ll_cid'] = 3
        elif item['c1title'] == u'经济':
            item['ll_cid'] = 4
        elif item['c1title'] == u'国际':
            item['ll_cid'] = 5
        elif item['c1title'] == u'调查':
            item['ll_cid'] = 6
        elif item['c1title'] == u'人物':
            item['ll_cid'] = 7
        elif item['c1title'] == u'汽车':
            item['ll_cid'] = 8
        else:
            item['ll_cid'] = 2

        item['ll_rnum'] = 0
        item['ll_uid'] = 1

        yield item







